{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "import tensorflow as tf\n",
    "import levenberg_marquardt as lm\n",
    "import pandas as pd\n",
    "from tensorflow.keras.experimental import WideDeepModel,LinearModel\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler,MinMaxScaler\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "from sklearn.mixture import GaussianMixture\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.gaussian_process import GaussianProcessRegressor,kernels\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense,BatchNormalization,LSTM,Input\n",
    "from keras import optimizers,regularizers,initializers\n",
    "from keras.optimizers import SGD,Adam\n",
    "from keras.losses import MeanSquaredError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def import_dataset(normalised=True,scaleMethod='Standard'):\n",
    "    '''\n",
    "    Imports Dataset and returns either scaled values depending upon user inputs\n",
    "    \n",
    "    Input:\n",
    "        normalised -- boolean depending upon whether the user wants to scale the values\n",
    "        scaleMethod -- Type of scaler to be used if normalised is True\n",
    "    \n",
    "    Output:\n",
    "        (X_train,X_test,Y_train,Y_test) -- the training and testing dataset\n",
    "        scaler -- used to perform inverse transform if dataset is scaled\n",
    "    '''\n",
    "    data = pd.read_csv('Dataset/Static_Model/15000DwithQuat.csv')\n",
    "    dataS = data.drop('Unnamed: 0',axis=1)\n",
    "    \n",
    "    if normalised == False:\n",
    "        scaler = 'None'\n",
    "        X = dataS.iloc[:,:7].values\n",
    "        Y = dataS.iloc[:,7:].values\n",
    "        X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size=0.15,random_state=0)\n",
    "        \n",
    "    elif scaleMethod == 'Standard':\n",
    "        scaler = StandardScaler()\n",
    "        scaler.fit(dataS)\n",
    "        dataS = scaler.transform(dataS)\n",
    "        X = dataS[:,:7]\n",
    "        Y = dataS[:,7:]\n",
    "        X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size=0.15,random_state=0)\n",
    "    \n",
    "    elif scaleMethod == 'MinMax':\n",
    "        scaler = MinMaxScaler(feature_range=(0,1))\n",
    "        scaler.fit(dataS)\n",
    "        dataS = scaler.transform(dataS)\n",
    "        X = dataS[:,:7]\n",
    "        Y = dataS[:,7:]\n",
    "        X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size=0.15,random_state=0)\n",
    "    \n",
    "    return X_train,X_test,Y_train,Y_test,scaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inverseTransform(scaler,*arr):\n",
    "    '''\n",
    "    Used to perform Inverse Transformation on normalised dataset\n",
    "    \n",
    "    Input:\n",
    "        scaler -- Instance of Normaliser used\n",
    "        *arr -- list of arrays to be concatenated\n",
    "    '''\n",
    "    data = np.concatenate(arr,axis=1)\n",
    "    data = pd.DataFrame(data)\n",
    "    arrInverse = scaler.inverse_transform(data)\n",
    "    \n",
    "    return arrInverse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost(y_test,y_pred):\n",
    "    '''\n",
    "    Calculates error of the model\n",
    "    '''\n",
    "    error = (y_test-y_pred)/y_test\n",
    "    error = np.sum(abs(error))/(y_test.shape[0]*y_test.shape[1])*100\n",
    "    \n",
    "    return error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmse(y_test,y_pred):\n",
    "    error = np.sum((y_test-y_pred)**2)\n",
    "    error = error/(y_test.shape[0]*y_test.shape[1])\n",
    "    error = math.sqrt(error)\n",
    "    return error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def errorMagnitude(y_true,y_pred):\n",
    "    \n",
    "    minMag = min([min(abs(i)) for i in y_true-y_pred])\n",
    "    maxMag = max([max(abs(i)) for i in y_true-y_pred])\n",
    "    \n",
    "    return (minMag,maxMag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(X_train,Y_train,X_test,Y_test,Model,normalise,scaler,giveMagnitude=False):\n",
    "    \n",
    "    Model.fit(X_train,Y_train)\n",
    "    Ytr_pred = Model.predict(X_train)\n",
    "    Yts_pred = Model.predict(X_test)\n",
    "\n",
    "    if not normalise:\n",
    "        error = rmse(Y_test,Yts_pred)\n",
    "        error_tr = rmse(Y_train,Ytr_pred)\n",
    "        \n",
    "        if giveMagnitude:\n",
    "            magnitude = errorMagnitude(Y_test,Yts_pred)\n",
    "            return (error_tr,error,magnitude)\n",
    "    else:\n",
    "        true = inverseTransform(scaler,X_train,Y_train)\n",
    "        pred = inverseTransform(scaler,X_train,Ytr_pred)\n",
    "\n",
    "        error_tr = rmse(true[:,7:],pred[:,7:])\n",
    "\n",
    "        true = inverseTransform(scaler,X_test,Y_test)\n",
    "        pred = inverseTransform(scaler,X_test,Yts_pred)\n",
    "\n",
    "        error = rmse(true[:,7:],pred[:,7:])\n",
    "        \n",
    "        if giveMagnitude:\n",
    "            magnitude = errorMagnitude(true[:,7:],pred[:,7:])\n",
    "            return (error_tr,error,magnitude)\n",
    "        \n",
    "    return (error_tr,error)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K Neighbors Regressor "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "Rscore = {}\n",
    "for normalise in [True,False]:\n",
    "    X_train,X_test,Y_train,Y_test,scaler = import_dataset(normalised=normalise)\n",
    "    for nbr in range(2,100):\n",
    "        for wgt in ['uniform', 'distance']:\n",
    "            Model = KNeighborsRegressor(n_neighbors=nbr,weights=wgt)\n",
    "            neterror = train(X_train,Y_train,X_test,Y_test,Model,normalise,scaler)\n",
    "            param = {normalise,nbr,wgt}\n",
    "            Rscore[neterror]=param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(0.7803688167758105, 1.5523995762832596): {2, True, 'uniform'},\n",
       " (0.0, 1.4408686928205925): {2, True, 'distance'},\n",
       " (1.0451631514131634, 1.6384123600999827): {3, True, 'uniform'},\n",
       " (3.4286402952018376e-17, 1.4439093153879956): {3, True, 'distance'},\n",
       " (1.2343421285024985, 1.7276810887558052): {4, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.4622952095418074): {4, True, 'distance'},\n",
       " (1.3791435250709398, 1.8001419697099201): {5, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.480483400126304): {5, True, 'distance'},\n",
       " (1.506512189142477, 1.869713122512068): {6, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.4993360573729038): {6, True, 'distance'},\n",
       " (1.621384086064069, 1.944557495903704): {7, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.5165711431367799): {7, True, 'distance'},\n",
       " (1.7220040052559829, 2.016396762241444): {8, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.5362813365469359): {8, True, 'distance'},\n",
       " (1.8086732593505683, 2.0854075271236105): {9, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.554125152200175): {9, True, 'distance'},\n",
       " (1.8837718212431684, 2.1521175618446127): {10, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.5741714656377188): {10, True, 'distance'},\n",
       " (1.9533291410897864, 2.2024706470987296): {11, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.5886874528662613): {11, True, 'distance'},\n",
       " (2.0186297702247122, 2.248239880824196): {12, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.6018223287358357): {12, True, 'distance'},\n",
       " (2.0792059405165335, 2.2967030367293377): {13, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.618709303255771): {13, True, 'distance'},\n",
       " (2.13528105177469, 2.337310871245744): {14, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.6311987656417302): {14, True, 'distance'},\n",
       " (2.18733918798881, 2.3771169382829287): {15, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.64264872963176): {15, True, 'distance'},\n",
       " (2.232775042479342, 2.4087835687852905): {16, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.6511396042225295): {16, True, 'distance'},\n",
       " (2.27419790646785, 2.4434697473056): {17, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.6594613523230903): {17, True, 'distance'},\n",
       " (2.313452287453304, 2.479257087606525): {18, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.6675355022324432): {18, True, 'distance'},\n",
       " (2.3505353208591626, 2.5072859051240757): {19, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.6737050451104236): {19, True, 'distance'},\n",
       " (2.384924352399184, 2.535330353228155): {20, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.680285051052025): {20, True, 'distance'},\n",
       " (2.4138382762453916, 2.5607197648875433): {21, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.6858016043503974): {21, True, 'distance'},\n",
       " (2.4397468329015983, 2.5812747952148345): {22, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.690769089254767): {22, True, 'distance'},\n",
       " (2.4638954131900337, 2.6002042153978477): {23, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.6950114264740368): {23, True, 'distance'},\n",
       " (2.484953262164424, 2.616663201814172): {24, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.6981419242249236): {24, True, 'distance'},\n",
       " (2.504348631997597, 2.6293718726046427): {25, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.700572477214967): {25, True, 'distance'},\n",
       " (2.5233241300463947, 2.64235876236741): {26, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7025627976246838): {26, True, 'distance'},\n",
       " (2.538490362907563, 2.652015766136738): {27, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.704265581149849): {27, True, 'distance'},\n",
       " (2.5538122450158935, 2.6608753812955204): {28, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7054363486974016): {28, True, 'distance'},\n",
       " (2.5666776185973177, 2.6692664200175225): {29, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7067601353711557): {29, True, 'distance'},\n",
       " (2.5785149455876217, 2.675978210433719): {30, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7075684387429784): {30, True, 'distance'},\n",
       " (2.5887884366572043, 2.6830412109955684): {31, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7082780287013546): {31, True, 'distance'},\n",
       " (2.598183073052554, 2.6918611860505135): {32, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7092720392446517): {32, True, 'distance'},\n",
       " (2.606721343921733, 2.6985919533642293): {33, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7100624600247765): {33, True, 'distance'},\n",
       " (2.615572616714137, 2.7061423930177746): {34, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7106326334001067): {34, True, 'distance'},\n",
       " (2.6208352788803055, 2.7102890040106002): {35, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7112036111946851): {35, True, 'distance'},\n",
       " (2.625901414087071, 2.7143875194275675): {36, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7117294648008832): {36, True, 'distance'},\n",
       " (2.631249368255283, 2.718014927421523): {37, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7121009831095848): {37, True, 'distance'},\n",
       " (2.63508173226211, 2.7197296771507258): {38, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7125163901300773): {38, True, 'distance'},\n",
       " (2.636378251753405, 2.719160859631833): {39, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.712586054508582): {39, True, 'distance'},\n",
       " (2.6374341711885325, 2.7194816703841846): {40, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.712764555120493): {40, True, 'distance'},\n",
       " (2.63939696909661, 2.7199425449717958): {41, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7129898161902295): {41, True, 'distance'},\n",
       " (2.6399255428207713, 2.718312732535933): {42, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7130247813427397): {42, True, 'distance'},\n",
       " (2.6407798277806784, 2.718817064090761): {43, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7129396655974773): {43, True, 'distance'},\n",
       " (2.641664767582707, 2.7189368292835576): {44, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7129026882393719): {44, True, 'distance'},\n",
       " (2.6419120387567254, 2.718352048591629): {45, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7129452772180302): {45, True, 'distance'},\n",
       " (2.6429419721326353, 2.718715802813367): {46, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7129614265406843): {46, True, 'distance'},\n",
       " (2.643018229705247, 2.718014160339502): {47, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7131015383790271): {47, True, 'distance'},\n",
       " (2.643259724661573, 2.7158758754466916): {48, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7130692773310072): {48, True, 'distance'},\n",
       " (2.6438162606455338, 2.71487112845366): {49, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7129923624235057): {49, True, 'distance'},\n",
       " (2.644177548842725, 2.7132301782193124): {50, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7128854152501787): {50, True, 'distance'},\n",
       " (2.644483998405779, 2.71155489841284): {51, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.712714358000724): {51, True, 'distance'},\n",
       " (2.6439606553384274, 2.711621399592346): {52, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.712698181907059): {52, True, 'distance'},\n",
       " (2.643532511050823, 2.7111190254212714): {53, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.712748715467137): {53, True, 'distance'},\n",
       " (2.6439586281685874, 2.7117041630806433): {54, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7129475848222655): {54, True, 'distance'},\n",
       " (2.6435044211606984, 2.711503989356741): {55, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7131270385591035): {55, True, 'distance'},\n",
       " (2.642864153929303, 2.711651669384749): {56, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.71333015424185): {56, True, 'distance'},\n",
       " (2.642298603780124, 2.711022103701161): {57, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.713432784703394): {57, True, 'distance'},\n",
       " (2.641951454213166, 2.7111776490939308): {58, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7135211940434691): {58, True, 'distance'},\n",
       " (2.641774708456934, 2.710911797139978): {59, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7135578020096605): {59, True, 'distance'},\n",
       " (2.641735562285853, 2.7101606655940778): {60, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7136480415339492): {60, True, 'distance'},\n",
       " (2.6412161235808194, 2.708390496161477): {61, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7136281634131665): {61, True, 'distance'},\n",
       " (2.640563867569917, 2.7061145718256125): {62, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7135571355642807): {62, True, 'distance'},\n",
       " (2.640744925986403, 2.7053698518142397): {63, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.713437002182459): {63, True, 'distance'},\n",
       " (2.640476229275947, 2.704883986339764): {64, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.713369289094841): {64, True, 'distance'},\n",
       " (2.6401348712327386, 2.702754647104646): {65, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7133133114041297): {65, True, 'distance'},\n",
       " (2.6399914631240304, 2.7027794930520033): {66, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7133437272290353): {66, True, 'distance'},\n",
       " (2.6397402933640297, 2.702290400123356): {67, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7133640331505264): {67, True, 'distance'},\n",
       " (2.638682275279554, 2.700952919685202): {68, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7133018213699676): {68, True, 'distance'},\n",
       " (2.6383732381359115, 2.700216755941491): {69, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.713258778191248): {69, True, 'distance'},\n",
       " (2.637907509828987, 2.699746649741252): {70, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.713237254736496): {70, True, 'distance'},\n",
       " (2.638203008655987, 2.7000110656176743): {71, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7132435368131491): {71, True, 'distance'},\n",
       " (2.6374847083964172, 2.6980125922378124): {72, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7132350509398582): {72, True, 'distance'},\n",
       " (2.6367068993957856, 2.69631502190643): {73, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.713264144086575): {73, True, 'distance'},\n",
       " (2.635816186636699, 2.6945469474794996): {74, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.713160160086626): {74, True, 'distance'},\n",
       " (2.635210679857137, 2.693577612684579): {75, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7131198744492269): {75, True, 'distance'},\n",
       " (2.6347929554145884, 2.6927090230633985): {76, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7131075677619885): {76, True, 'distance'},\n",
       " (2.634627856755858, 2.6931779227709463): {77, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.713179103939318): {77, True, 'distance'},\n",
       " (2.63395814385938, 2.6925808266011195): {78, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7131854140319125): {78, True, 'distance'},\n",
       " (2.6334102258613736, 2.6918260671352856): {79, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7131878081508891): {79, True, 'distance'},\n",
       " (2.6330097755118906, 2.6909164976309135): {80, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7132010513356943): {80, True, 'distance'},\n",
       " (2.6332083034540523, 2.6908539314728617): {81, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.713242697178084): {81, True, 'distance'},\n",
       " (2.6332092709263395, 2.6905996498258165): {82, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7132888103043264): {82, True, 'distance'},\n",
       " (2.633166376514449, 2.6899976583291108): {83, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7133404608389378): {83, True, 'distance'},\n",
       " (2.6332158913336654, 2.690101219948827): {84, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7134217594048162): {84, True, 'distance'},\n",
       " (2.6329299171144154, 2.689533200215873): {85, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.713592700300948): {85, True, 'distance'},\n",
       " (2.63303487237796, 2.6891724933511525): {86, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7136574089314522): {86, True, 'distance'},\n",
       " (2.632843698217069, 2.6893965392892554): {87, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7137046885701004): {87, True, 'distance'},\n",
       " (2.632781765237553, 2.6892014949285574): {88, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7137729862991933): {88, True, 'distance'},\n",
       " (2.6332213983362767, 2.6893577133126865): {89, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.713853465715098): {89, True, 'distance'},\n",
       " (2.633971158150838, 2.6896414083200373): {90, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7138624379491902): {90, True, 'distance'},\n",
       " (2.634055861213588, 2.6896936027229197): {91, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7139329028656984): {91, True, 'distance'},\n",
       " (2.634426702762065, 2.690177526177082): {92, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7139685205189514): {92, True, 'distance'},\n",
       " (2.6346550457645823, 2.69029110026879): {93, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7139968061926683): {93, True, 'distance'},\n",
       " (2.6350041513717044, 2.6909827894881198): {94, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7140976574024789): {94, True, 'distance'},\n",
       " (2.635950299168415, 2.6918302283864244): {95, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7142053675002684): {95, True, 'distance'},\n",
       " (2.636148598033311, 2.691479398889934): {96, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.714237679528335): {96, True, 'distance'},\n",
       " (2.636731715089949, 2.6922870150936378): {97, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7142589977952103): {97, True, 'distance'},\n",
       " (2.637257085239687, 2.692593222668602): {98, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7142453142457181): {98, True, 'distance'},\n",
       " (2.63768673928837, 2.693130840405447): {99, True, 'uniform'},\n",
       " (3.337193407403036e-17, 1.7142895442061967): {99, True, 'distance'},\n",
       " (1.0315797898103172, 1.9785937767347124): {2, False, 'uniform'},\n",
       " (0.0, 2.0019398915717526): {2, False, 'distance'},\n",
       " (1.3214428966005436, 1.9458122173491421): {3, False, 'uniform'},\n",
       " (0.0, 1.9511098513400411): {3, False, 'distance'},\n",
       " (1.4695504181117942, 1.934879051632025): {4, False, 'uniform'},\n",
       " (0.0, 1.9401301368624013): {4, False, 'distance'},\n",
       " (1.5576500651591152, 1.9471186689853062): {5, False, 'uniform'},\n",
       " (0.0, 1.9460798217356619): {5, False, 'distance'},\n",
       " (1.6122513994918408, 1.9548720396378954): {6, False, 'uniform'},\n",
       " (0.0, 1.9471412413910967): {6, False, 'distance'},\n",
       " (1.6549721366564585, 1.9514876004923227): {7, False, 'uniform'},\n",
       " (0.0, 1.9467628334258102): {7, False, 'distance'},\n",
       " (1.6927229448841459, 1.95419909354759): {8, False, 'uniform'},\n",
       " (0.0, 1.9481794077810008): {8, False, 'distance'},\n",
       " (1.7256389608743017, 1.961577778211347): {9, False, 'uniform'},\n",
       " (0.0, 1.9506173329078764): {9, False, 'distance'},\n",
       " (1.7583032708144697, 1.9822265819583347): {10, False, 'uniform'},\n",
       " (0.0, 1.9537807659885935): {10, False, 'distance'},\n",
       " (1.7905601513172726, 1.9949030553346834): {11, False, 'uniform'},\n",
       " (0.0, 1.9562748587601917): {11, False, 'distance'},\n",
       " (1.822989312277979, 2.017171462686438): {12, False, 'uniform'},\n",
       " (0.0, 1.9587250819974287): {12, False, 'distance'},\n",
       " (1.8537152529465912, 2.0377734687788953): {13, False, 'uniform'},\n",
       " (0.0, 1.9613075830967837): {13, False, 'distance'},\n",
       " (1.8842647588575732, 2.059909405136453): {14, False, 'uniform'},\n",
       " (0.0, 1.9641723277276273): {14, False, 'distance'},\n",
       " (1.9138872828853881, 2.082695665374061): {15, False, 'uniform'},\n",
       " (0.0, 1.9673737163716216): {15, False, 'distance'},\n",
       " (1.9448248635802659, 2.1073499023918907): {16, False, 'uniform'},\n",
       " (0.0, 1.970104709940791): {16, False, 'distance'},\n",
       " (1.9710604023836078, 2.1264259751781487): {17, False, 'uniform'},\n",
       " (0.0, 1.9723426511020912): {17, False, 'distance'},\n",
       " (1.9991102020891267, 2.1470981629310764): {18, False, 'uniform'},\n",
       " (0.0, 1.9745825913982853): {18, False, 'distance'},\n",
       " (2.025977214285566, 2.166490865799175): {19, False, 'uniform'},\n",
       " (0.0, 1.9762685599837873): {19, False, 'distance'},\n",
       " (2.053100751565405, 2.185615982331348): {20, False, 'uniform'},\n",
       " (0.0, 1.9785075557981222): {20, False, 'distance'},\n",
       " (2.0763112335212393, 2.2056848948522063): {21, False, 'uniform'},\n",
       " (0.0, 1.9806649888941994): {21, False, 'distance'},\n",
       " (2.1000754475570296, 2.2275944933523224): {22, False, 'uniform'},\n",
       " (0.0, 1.9825496059935888): {22, False, 'distance'},\n",
       " (2.122411803071791, 2.249957314818331): {23, False, 'uniform'},\n",
       " (0.0, 1.9845780381448823): {23, False, 'distance'},\n",
       " (2.144569333921299, 2.2679983577873077): {24, False, 'uniform'},\n",
       " (0.0, 1.986117532542231): {24, False, 'distance'},\n",
       " (2.1690567788335837, 2.2898938160728957): {25, False, 'uniform'},\n",
       " (0.0, 1.988202080426684): {25, False, 'distance'},\n",
       " (2.191103715335334, 2.3103814104320426): {26, False, 'uniform'},\n",
       " (0.0, 1.9901412605352105): {26, False, 'distance'},\n",
       " (2.212962285363445, 2.330746165142893): {27, False, 'uniform'},\n",
       " (0.0, 1.991996496198236): {27, False, 'distance'},\n",
       " (2.235115158310274, 2.349354949584741): {28, False, 'uniform'},\n",
       " (0.0, 1.9936909051713365): {28, False, 'distance'},\n",
       " (2.255368276950701, 2.3642733232868207): {29, False, 'uniform'},\n",
       " (0.0, 1.995047652534929): {29, False, 'distance'},\n",
       " (2.276132876494625, 2.382562786601597): {30, False, 'uniform'},\n",
       " (0.0, 1.996704851266486): {30, False, 'distance'},\n",
       " (2.296602001057197, 2.397863445641276): {31, False, 'uniform'},\n",
       " (0.0, 1.9980812954025335): {31, False, 'distance'},\n",
       " (2.3158305209661254, 2.4128193828241566): {32, False, 'uniform'},\n",
       " (0.0, 1.999480522131334): {32, False, 'distance'},\n",
       " (2.3349629237854987, 2.427525134314115): {33, False, 'uniform'},\n",
       " (0.0, 2.000877784859752): {33, False, 'distance'},\n",
       " (2.352646265399535, 2.4436437842394727): {34, False, 'uniform'},\n",
       " (0.0, 2.002350597513257): {34, False, 'distance'},\n",
       " (2.370142798810802, 2.4592511995697053): {35, False, 'uniform'},\n",
       " (0.0, 2.003646211669764): {35, False, 'distance'},\n",
       " (2.3873048340361653, 2.474125741032921): {36, False, 'uniform'},\n",
       " (0.0, 2.0048715673410693): {36, False, 'distance'},\n",
       " (2.404351695157271, 2.4884775942236588): {37, False, 'uniform'},\n",
       " (0.0, 2.0060302815463085): {37, False, 'distance'},\n",
       " (2.421092475106255, 2.50417181927712): {38, False, 'uniform'},\n",
       " (0.0, 2.0072043008699874): {38, False, 'distance'},\n",
       " (2.4380367573789616, 2.5207048238190453): {39, False, 'uniform'},\n",
       " (0.0, 2.0082706444708704): {39, False, 'distance'},\n",
       " (2.45465292213324, 2.5360739150856344): {40, False, 'uniform'},\n",
       " (0.0, 2.009092562764382): {40, False, 'distance'},\n",
       " (2.4695987574203726, 2.549913675557008): {41, False, 'uniform'},\n",
       " (0.0, 2.0099513823677535): {41, False, 'distance'},\n",
       " (2.4840156490265826, 2.5628048614503447): {42, False, 'uniform'},\n",
       " (0.0, 2.0107832261845466): {42, False, 'distance'},\n",
       " (2.4982348024297774, 2.575144304113459): {43, False, 'uniform'},\n",
       " (0.0, 2.01160965755898): {43, False, 'distance'},\n",
       " (2.511109844735071, 2.5867374155180767): {44, False, 'uniform'},\n",
       " (0.0, 2.0124605340656436): {44, False, 'distance'},\n",
       " (2.5239273247547978, 2.5990795563942233): {45, False, 'uniform'},\n",
       " (0.0, 2.013353437371337): {45, False, 'distance'},\n",
       " (2.536597654784028, 2.609162770175261): {46, False, 'uniform'},\n",
       " (0.0, 2.014074720686902): {46, False, 'distance'},\n",
       " (2.5478221841612894, 2.6196510072582155): {47, False, 'uniform'},\n",
       " (0.0, 2.014775637281387): {47, False, 'distance'},\n",
       " (2.558382285942421, 2.630017251034818): {48, False, 'uniform'},\n",
       " (0.0, 2.0154196452116535): {48, False, 'distance'},\n",
       " (2.569155964881761, 2.6401711457825283): {49, False, 'uniform'},\n",
       " (0.0, 2.016081166171032): {49, False, 'distance'},\n",
       " (2.578747564986067, 2.649960209344871): {50, False, 'uniform'},\n",
       " (0.0, 2.0166172261866): {50, False, 'distance'},\n",
       " (2.587445920193395, 2.658567694518721): {51, False, 'uniform'},\n",
       " (0.0, 2.01708750555414): {51, False, 'distance'},\n",
       " (2.595140994258095, 2.66467425876512): {52, False, 'uniform'},\n",
       " (0.0, 2.017486236420277): {52, False, 'distance'},\n",
       " (2.6023778744636576, 2.6719441666180352): {53, False, 'uniform'},\n",
       " (0.0, 2.0178834337430063): {53, False, 'distance'},\n",
       " (2.609141800517051, 2.67944362745713): {54, False, 'uniform'},\n",
       " (0.0, 2.018298940868898): {54, False, 'distance'},\n",
       " (2.6157460607293967, 2.686045468177451): {55, False, 'uniform'},\n",
       " (0.0, 2.018722231029788): {55, False, 'distance'},\n",
       " (2.6218457797164665, 2.6914130796069498): {56, False, 'uniform'},\n",
       " (0.0, 2.0191049418948657): {56, False, 'distance'},\n",
       " (2.6276161482626943, 2.6973678029762045): {57, False, 'uniform'},\n",
       " (0.0, 2.019467312235276): {57, False, 'distance'},\n",
       " (2.6327642363029917, 2.7037149530777773): {58, False, 'uniform'},\n",
       " (0.0, 2.0198305613547363): {58, False, 'distance'},\n",
       " (2.6377866068549207, 2.7094252389764284): {59, False, 'uniform'},\n",
       " (0.0, 2.0202093850045637): {59, False, 'distance'},\n",
       " (2.6422305762935703, 2.713546566605372): {60, False, 'uniform'},\n",
       " (0.0, 2.020413126823851): {60, False, 'distance'},\n",
       " (2.644903476298039, 2.7165959319621362): {61, False, 'uniform'},\n",
       " (0.0, 2.020605010499734): {61, False, 'distance'},\n",
       " (2.647526166863629, 2.719714869426445): {62, False, 'uniform'},\n",
       " (0.0, 2.020801706003488): {62, False, 'distance'},\n",
       " (2.6501844703516824, 2.7218781205897833): {63, False, 'uniform'},\n",
       " (0.0, 2.020952376399746): {63, False, 'distance'},\n",
       " (2.6522856987604606, 2.7234617138115276): {64, False, 'uniform'},\n",
       " (0.0, 2.0210820239745133): {64, False, 'distance'},\n",
       " (2.6540300755993456, 2.7251259228154074): {65, False, 'uniform'},\n",
       " (0.0, 2.021207619341419): {65, False, 'distance'},\n",
       " (2.655374588684892, 2.726457804735657): {66, False, 'uniform'},\n",
       " (0.0, 2.021316355313229): {66, False, 'distance'},\n",
       " (2.656599983309058, 2.727456050881909): {67, False, 'uniform'},\n",
       " (0.0, 2.021430766348907): {67, False, 'distance'},\n",
       " (2.657871005862563, 2.7285118054415083): {68, False, 'uniform'},\n",
       " (0.0, 2.0215390951690493): {68, False, 'distance'},\n",
       " (2.658404641467798, 2.7290550283060795): {69, False, 'uniform'},\n",
       " (0.0, 2.0216306628852467): {69, False, 'distance'},\n",
       " (2.659383217898439, 2.7295584602193634): {70, False, 'uniform'},\n",
       " (0.0, 2.0217239180852227): {70, False, 'distance'},\n",
       " (2.659638386710405, 2.7298584056419033): {71, False, 'uniform'},\n",
       " (0.0, 2.0218123446014684): {71, False, 'distance'},\n",
       " (2.660424541226576, 2.7298014528940056): {72, False, 'uniform'},\n",
       " (0.0, 2.021903107635915): {72, False, 'distance'},\n",
       " (2.6607311594339587, 2.7293653603583024): {73, False, 'uniform'},\n",
       " (0.0, 2.021926988465264): {73, False, 'distance'},\n",
       " (2.6605848708393425, 2.7293779506034856): {74, False, 'uniform'},\n",
       " (0.0, 2.0219573640581956): {74, False, 'distance'},\n",
       " (2.6606151368279636, 2.7287816337952493): {75, False, 'uniform'},\n",
       " (0.0, 2.0220158209968275): {75, False, 'distance'},\n",
       " (2.6597163258679144, 2.727273251909628): {76, False, 'uniform'},\n",
       " (0.0, 2.022043874819819): {76, False, 'distance'},\n",
       " (2.6592059478015573, 2.7265265522222615): {77, False, 'uniform'},\n",
       " (0.0, 2.0220691289818946): {77, False, 'distance'},\n",
       " (2.658543136783342, 2.7244310900460262): {78, False, 'uniform'},\n",
       " (0.0, 2.0220852435390375): {78, False, 'distance'},\n",
       " (2.6573472285266617, 2.7224887390103834): {79, False, 'uniform'},\n",
       " (0.0, 2.0220954327500973): {79, False, 'distance'},\n",
       " (2.6562361937357464, 2.721322627010215): {80, False, 'uniform'},\n",
       " (0.0, 2.0221032699622126): {80, False, 'distance'},\n",
       " (2.6549937092602915, 2.7208700672164388): {81, False, 'uniform'},\n",
       " (0.0, 2.0221010493600318): {81, False, 'distance'},\n",
       " (2.654135651356662, 2.7194897886957174): {82, False, 'uniform'},\n",
       " (0.0, 2.0221009521822224): {82, False, 'distance'},\n",
       " (2.653168789582839, 2.7179793419077063): {83, False, 'uniform'},\n",
       " (0.0, 2.0221017346812302): {83, False, 'distance'},\n",
       " (2.6522209758778397, 2.717560956149931): {84, False, 'uniform'},\n",
       " (0.0, 2.0220866642997426): {84, False, 'distance'},\n",
       " (2.6511134959043323, 2.7160437847778613): {85, False, 'uniform'},\n",
       " (0.0, 2.0220674246601495): {85, False, 'distance'},\n",
       " (2.650161990804236, 2.714485474144148): {86, False, 'uniform'},\n",
       " (0.0, 2.0220525800743245): {86, False, 'distance'},\n",
       " (2.649217140852015, 2.713386030097851): {87, False, 'uniform'},\n",
       " (0.0, 2.0220368858412314): {87, False, 'distance'},\n",
       " (2.6487402600214196, 2.7129525902183156): {88, False, 'uniform'},\n",
       " (0.0, 2.0220403185396565): {88, False, 'distance'},\n",
       " (2.6480266738885962, 2.712307210241297): {89, False, 'uniform'},\n",
       " (0.0, 2.022041301664342): {89, False, 'distance'},\n",
       " (2.647390716060974, 2.711799442066424): {90, False, 'uniform'},\n",
       " (0.0, 2.0220392812312404): {90, False, 'distance'},\n",
       " (2.647001437691619, 2.7111734837925625): {91, False, 'uniform'},\n",
       " (0.0, 2.0220371504527463): {91, False, 'distance'},\n",
       " (2.646510780064875, 2.710572541441284): {92, False, 'uniform'},\n",
       " (0.0, 2.0220310309715366): {92, False, 'distance'},\n",
       " (2.6457030992666692, 2.7104187295499664): {93, False, 'uniform'},\n",
       " (0.0, 2.0220298359424): {93, False, 'distance'},\n",
       " (2.645406541833631, 2.709788924119185): {94, False, 'uniform'},\n",
       " (0.0, 2.02203208675381): {94, False, 'distance'},\n",
       " (2.645049017305488, 2.708752142914732): {95, False, 'uniform'},\n",
       " (0.0, 2.0220298789449034): {95, False, 'distance'},\n",
       " (2.6445991823250923, 2.7073704640785055): {96, False, 'uniform'},\n",
       " (0.0, 2.0220290792117708): {96, False, 'distance'},\n",
       " (2.644471682230614, 2.7061771057523316): {97, False, 'uniform'},\n",
       " (0.0, 2.0220312911590703): {97, False, 'distance'},\n",
       " (2.644110641704898, 2.7054610502079375): {98, False, 'uniform'},\n",
       " (0.0, 2.022032832675679): {98, False, 'distance'},\n",
       " (2.643652085867698, 2.7043102560419525): {99, False, 'uniform'},\n",
       " (0.0, 2.0220407577209283): {99, False, 'distance'}}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Rscore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.506512189142477, 1.869713122512068, (0.0, 7.833333333333334))"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nbr = 6\n",
    "wgt = 'uniform'\n",
    "normalise = True\n",
    "X_train,X_test,Y_train,Y_test,scaler = import_dataset(normalised=normalise)\n",
    "Model = KNeighborsRegressor(n_neighbors=nbr,weights=wgt)\n",
    "neterror = train(X_train,Y_train,X_test,Y_test,Model,normalise,scaler,giveMagnitude=True)\n",
    "neterror"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "Rscore = {}\n",
    "for normalise in [True,False]:\n",
    "    X_train,X_test,Y_train,Y_test,scaler = import_dataset(normalised=normalise,scaleMethod='MinMax')\n",
    "    for ctr in ['mse','friedman_mse','mae']:\n",
    "        for spt in ['best', 'random']:\n",
    "            for sampsplit in [2,5,7]:\n",
    "                for leaf in [1,3,5]:\n",
    "                    Model = DecisionTreeRegressor(criterion=ctr,splitter=spt,min_samples_split=sampsplit,min_samples_leaf=leaf)\n",
    "                    neterror = train(X_train,Y_train,X_test,Y_test,Model,normalise,scaler)\n",
    "                    param = {normalise,ctr,spt,sampsplit,leaf}\n",
    "                    Rscore[neterror]=param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(0.18226246965060375, 1.6930285523322828): {2, True, 'best', 'mse'},\n",
       " (0.9743475635537735, 1.8263460592651928): {2, 3, True, 'best', 'mse'},\n",
       " (1.2830038362599183, 1.9381489713739086): {2, 5, True, 'best', 'mse'},\n",
       " (0.6415583050849933, 1.7464202805442612): {5, True, 'best', 'mse'},\n",
       " (0.9743475635537735, 1.8288602727735386): {3, 5, True, 'best', 'mse'},\n",
       " (1.2830038362599185, 1.9383714249123547): {5, True, 'best', 'mse'},\n",
       " (0.8600395187758578, 1.8015213906550358): {7, True, 'best', 'mse'},\n",
       " (1.0235240791580071, 1.8459115456506412): {3, 7, True, 'best', 'mse'},\n",
       " (1.2830038362599185, 1.939908907663553): {5, 7, True, 'best', 'mse'},\n",
       " (0.17969836746051385, 1.7639251964767193): {2, True, 'mse', 'random'},\n",
       " (1.43592958850484, 1.971756753218795): {2, 3, True, 'mse', 'random'},\n",
       " (1.7734887361593679, 2.1392310130050443): {2, 5, True, 'mse', 'random'},\n",
       " (1.004991603571542, 1.900773883981125): {5, True, 'mse', 'random'},\n",
       " (1.4434314987209482, 1.9785645085493893): {3, 5, True, 'mse', 'random'},\n",
       " (1.7652081319777069, 2.109860671282156): {5, True, 'mse', 'random'},\n",
       " (1.2973037650540946, 1.9248224304825856): {7, True, 'mse', 'random'},\n",
       " (1.485148466138884, 2.0669677472435724): {3, 7, True, 'mse', 'random'},\n",
       " (1.7791712191585562, 2.146255249254502): {5, 7, True, 'mse', 'random'},\n",
       " (0.18226246965060375, 2.8964985247197523): {2, True, 'best', 'friedman_mse'},\n",
       " (2.233293082542089, 3.059433797133891): {2, 3, True, 'best', 'friedman_mse'},\n",
       " (2.6178900363128665, 3.1328458669241184): {2,\n",
       "  5,\n",
       "  True,\n",
       "  'best',\n",
       "  'friedman_mse'},\n",
       " (1.8795365163624433, 3.0884748173703573): {5, True, 'best', 'friedman_mse'},\n",
       " (2.2439458023250416, 3.054599217481839): {3, 5, True, 'best', 'friedman_mse'},\n",
       " (2.6282848075280008, 3.1094793003162238): {5, True, 'best', 'friedman_mse'},\n",
       " (2.2316163285628865, 3.160707863363975): {7, True, 'best', 'friedman_mse'},\n",
       " (2.3147708120311954, 3.0434134107880375): {3,\n",
       "  7,\n",
       "  True,\n",
       "  'best',\n",
       "  'friedman_mse'},\n",
       " (2.6273167680512195, 3.124319382577177): {5, 7, True, 'best', 'friedman_mse'},\n",
       " (0.1812916618784953, 1.9120766294887444): {2, True, 'friedman_mse', 'random'},\n",
       " (1.5037667434874846, 2.0883152013952393): {2,\n",
       "  3,\n",
       "  True,\n",
       "  'friedman_mse',\n",
       "  'random'},\n",
       " (1.8599772046451528, 2.2155695418486743): {2,\n",
       "  5,\n",
       "  True,\n",
       "  'friedman_mse',\n",
       "  'random'},\n",
       " (1.1519707467237132, 1.9449979370962434): {5, True, 'friedman_mse', 'random'},\n",
       " (1.604089498709312, 2.166853122980438): {3,\n",
       "  5,\n",
       "  True,\n",
       "  'friedman_mse',\n",
       "  'random'},\n",
       " (1.8287510641563214, 2.1984064859297376): {5, True, 'friedman_mse', 'random'},\n",
       " (1.4392255769815614, 2.0337599582795494): {7, True, 'friedman_mse', 'random'},\n",
       " (1.5752270858893587, 2.1263065017550504): {3,\n",
       "  7,\n",
       "  True,\n",
       "  'friedman_mse',\n",
       "  'random'},\n",
       " (1.8515259080766726, 2.19021189252272): {5,\n",
       "  7,\n",
       "  True,\n",
       "  'friedman_mse',\n",
       "  'random'},\n",
       " (0.1987214031714535, 1.7210139130433806): {2, True, 'best', 'mae'},\n",
       " (1.1608862402255864, 1.973181300449719): {2, 3, True, 'best', 'mae'},\n",
       " (1.439158250710545, 2.0539934004005196): {2, 5, True, 'best', 'mae'},\n",
       " (0.7152402585502164, 1.822658315025977): {5, True, 'best', 'mae'},\n",
       " (1.1640657282163422, 1.9826189189498262): {3, 5, True, 'best', 'mae'},\n",
       " (1.440135478376403, 2.0581612613635936): {5, True, 'best', 'mae'},\n",
       " (0.9619934368534017, 1.8962755894413426): {7, True, 'best', 'mae'},\n",
       " (1.1922297449342063, 1.96742217126879): {3, 7, True, 'best', 'mae'},\n",
       " (1.4386608697115577, 2.0591462632201076): {5, 7, True, 'best', 'mae'},\n",
       " (0.18502781984784628, 1.760681686165901): {2, True, 'mae', 'random'},\n",
       " (1.5314641206835948, 2.166070430782691): {2, 3, True, 'mae', 'random'},\n",
       " (1.8803342422944644, 2.3142313338702047): {2, 5, True, 'mae', 'random'},\n",
       " (1.0069318570319852, 1.9091519350515587): {5, True, 'mae', 'random'},\n",
       " (1.5509484764074493, 2.1204100756431266): {3, 5, True, 'mae', 'random'},\n",
       " (1.8723059076349426, 2.3133909695029455): {5, True, 'mae', 'random'},\n",
       " (1.3834384557270025, 2.020416623702481): {7, True, 'mae', 'random'},\n",
       " (1.5650753286387609, 2.1114634208950385): {3, 7, True, 'mae', 'random'},\n",
       " (1.8718005382256842, 2.2527698999725256): {5, 7, True, 'mae', 'random'},\n",
       " (1.4023507775746316, 2.038816519528276): {1, 2, False, 'best', 'mse'},\n",
       " (1.5083098556531362, 1.9865229340737192): {2, 3, False, 'best', 'mse'},\n",
       " (1.6264383755772363, 2.0219409671448796): {2, 5, False, 'best', 'mse'},\n",
       " (1.436352653492254, 2.03429366916386): {1, 5, False, 'best', 'mse'},\n",
       " (1.5083098556531362, 1.9858698929684162): {3, 5, False, 'best', 'mse'},\n",
       " (1.6264383755772363, 2.0220824348636737): {5, False, 'best', 'mse'},\n",
       " (1.4799268410966, 2.022525234698631): {1, 7, False, 'best', 'mse'},\n",
       " (1.517218794052591, 1.9869054441880387): {3, 7, False, 'best', 'mse'},\n",
       " (1.6264383755772363, 2.0222472747047697): {5, 7, False, 'best', 'mse'},\n",
       " (1.4000738376897204, 2.0454333355091796): {1, 2, False, 'mse', 'random'},\n",
       " (1.6346345366647512, 2.032345916509138): {2, 3, False, 'mse', 'random'},\n",
       " (1.8187736872567355, 2.102725910938665): {2, 5, False, 'mse', 'random'},\n",
       " (1.449402668522808, 2.03071194717949): {1, 5, False, 'mse', 'random'},\n",
       " (1.6320078166742094, 2.0213202055711865): {3, 5, False, 'mse', 'random'},\n",
       " (1.8460517311695566, 2.1086004035672605): {5, False, 'mse', 'random'},\n",
       " (1.5184932302424923, 2.032910322349378): {1, 7, False, 'mse', 'random'},\n",
       " (1.6800164506976139, 2.0537438507353785): {3, 7, False, 'mse', 'random'},\n",
       " (1.8384376742117017, 2.147705863052101): {5, 7, False, 'mse', 'random'},\n",
       " (1.4023507775746316, 2.50861117836534): {1, 2, False, 'best', 'friedman_mse'},\n",
       " (2.0918594119760674, 2.7634124303798435): {2,\n",
       "  3,\n",
       "  False,\n",
       "  'best',\n",
       "  'friedman_mse'},\n",
       " (2.5005497054416104, 2.9849353887654755): {2,\n",
       "  5,\n",
       "  False,\n",
       "  'best',\n",
       "  'friedman_mse'},\n",
       " (1.8131359260603463, 2.671258029056681): {1,\n",
       "  5,\n",
       "  False,\n",
       "  'best',\n",
       "  'friedman_mse'},\n",
       " (2.094798384080892, 2.768340496990346): {3, 5, False, 'best', 'friedman_mse'},\n",
       " (2.4941977800697597, 2.9908760081565124): {5, False, 'best', 'friedman_mse'},\n",
       " (2.0864917526995232, 2.78305081758121): {1, 7, False, 'best', 'friedman_mse'},\n",
       " (2.1602226512105367, 2.823623842909277): {3,\n",
       "  7,\n",
       "  False,\n",
       "  'best',\n",
       "  'friedman_mse'},\n",
       " (2.499235578656521, 2.9865677162740405): {5,\n",
       "  7,\n",
       "  False,\n",
       "  'best',\n",
       "  'friedman_mse'},\n",
       " (1.402108466397463, 2.0681592401169366): {1,\n",
       "  2,\n",
       "  False,\n",
       "  'friedman_mse',\n",
       "  'random'},\n",
       " (1.7738288536217386, 2.248307783150274): {2,\n",
       "  3,\n",
       "  False,\n",
       "  'friedman_mse',\n",
       "  'random'},\n",
       " (1.8635564662578576, 2.1650919763287897): {2,\n",
       "  5,\n",
       "  False,\n",
       "  'friedman_mse',\n",
       "  'random'},\n",
       " (1.4980961089763856, 2.092312493190931): {1,\n",
       "  5,\n",
       "  False,\n",
       "  'friedman_mse',\n",
       "  'random'},\n",
       " (1.6755804936278407, 2.081061039086482): {3,\n",
       "  5,\n",
       "  False,\n",
       "  'friedman_mse',\n",
       "  'random'},\n",
       " (1.86572268738749, 2.1173122214397164): {5, False, 'friedman_mse', 'random'},\n",
       " (1.5986276688345429, 2.09255008660611): {1,\n",
       "  7,\n",
       "  False,\n",
       "  'friedman_mse',\n",
       "  'random'},\n",
       " (1.637952424873351, 2.0098410371613395): {3,\n",
       "  7,\n",
       "  False,\n",
       "  'friedman_mse',\n",
       "  'random'},\n",
       " (1.9141526498768122, 2.193610243699644): {5,\n",
       "  7,\n",
       "  False,\n",
       "  'friedman_mse',\n",
       "  'random'},\n",
       " (1.4830149471038157, 2.1382755253292833): {1, 2, False, 'best', 'mae'},\n",
       " (1.620058096271278, 2.1616865843338364): {2, 3, False, 'best', 'mae'},\n",
       " (1.7340873171583406, 2.125989965482747): {2, 5, False, 'best', 'mae'},\n",
       " (1.522127638872512, 2.1628492010925466): {1, 5, False, 'best', 'mae'},\n",
       " (1.6205482006371286, 2.153169498411328): {3, 5, False, 'best', 'mae'},\n",
       " (1.732554500741381, 2.1257155657958253): {5, False, 'best', 'mae'},\n",
       " (1.5728330876379182, 2.175060663266404): {1, 7, False, 'best', 'mae'},\n",
       " (1.625052035365963, 2.1539885484065757): {3, 7, False, 'best', 'mae'},\n",
       " (1.7330523905050574, 2.123401359454527): {5, 7, False, 'best', 'mae'},\n",
       " (1.4786055964333158, 2.131144710660864): {1, 2, False, 'mae', 'random'},\n",
       " (1.7332220929613757, 2.232344308369816): {2, 3, False, 'mae', 'random'},\n",
       " (1.9132458817207914, 2.2407092131237776): {2, 5, False, 'mae', 'random'},\n",
       " (1.5356482951100847, 2.15625060386465): {1, 5, False, 'mae', 'random'},\n",
       " (1.750582536096563, 2.2132178684741666): {3, 5, False, 'mae', 'random'},\n",
       " (1.9472529670402505, 2.262661903353855): {5, False, 'mae', 'random'},\n",
       " (1.6080557009180068, 2.1500129198578217): {1, 7, False, 'mae', 'random'},\n",
       " (1.7631967403265125, 2.1999116143861577): {3, 7, False, 'mae', 'random'},\n",
       " (1.930353518369719, 2.218633212888812): {5, 7, False, 'mae', 'random'}}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Rscore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,Y_train,Y_test,sci = import_dataset(normalised=True)\n",
    "Model = DecisionTreeRegressor(criterion='mae',splitter='best',min_samples_split=5,min_samples_leaf=3)\n",
    "values = train(X_train,Y_train,X_test,Y_test,Model,normalise=True,scaler=sci,giveMagnitude=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.1652778848925298, 1.9655434645693062, (0.0, 11.0))"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gaussian Mixture Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "Rscore = {}\n",
    "for normalise in [True,False]:\n",
    "    X_train,X_test,Y_train,Y_test,scaler = import_dataset(normalised=normalise)\n",
    "    for cpt in range(2,4):\n",
    "        for cvr in ['full', 'tied','diag','spherical']:\n",
    "            Model = GaussianMixture(n_components=cpt,covariance_type=cvr)\n",
    "            Model.fit(X_train,Y_train[:,[0]])\n",
    "            Ytr_pred = Model.predict(X_train)\n",
    "            Yts_pred = Model.predict(X_test)\n",
    "            Ytr_pred = Ytr_pred.reshape(Ytr_pred.shape[0],1)\n",
    "            Yts_pred = Yts_pred.reshape(Yts_pred.shape[0],1)\n",
    "            if normalise == False:\n",
    "                error = cost(Y_test[:,[0]],Yts_pred[:,[0]])\n",
    "                error_tr = cost(Y_train[:,[0]],Ytr_pred[:,[0]])\n",
    "            else:\n",
    "                true = inverseTransform(scaler,X_train,Y_train)\n",
    "                pred = inverseTransform(scaler,X_train,Ytr_pred,Y_train[:,1:])\n",
    "                \n",
    "                error_tr = cost(true[:,3:],pred[:,3:])\n",
    "                \n",
    "                true = inverseTransform(scaler,X_test,Y_test)\n",
    "                pred = inverseTransform(scaler,X_test,Yts_pred,Y_test[:,1:])\n",
    "                \n",
    "                error = cost(true[:,3:],pred[:,3:])\n",
    "#             valTst = Model.score(X_test,Y_test)\n",
    "#             val = Model.score(X_train,Y_train)\n",
    "            param = {normalise,cpt,cvr}\n",
    "            Rscore[(error,error_tr)]=param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(1.2660092323866023, 1.24709715142702): {2, True, 'full'},\n",
       " (1.3795775748341284, 1.3736157175060968): {2, True, 'tied'},\n",
       " (1.1258963890878102, 1.1369794626085483): {2, True, 'diag'},\n",
       " (1.3550035319133946, 1.3487678495161903): {2, True, 'spherical'},\n",
       " (1.6296353958208654, 1.6553384724433031): {3, True, 'full'},\n",
       " (1.3877418659182346, 1.4230583718113954): {3, True, 'tied'},\n",
       " (1.6117110666233059, 1.6281736313921367): {3, True, 'diag'},\n",
       " (1.7631838510860192, 1.7588711679611089): {3, True, 'spherical'},\n",
       " (99.84412447428427, 99.84298182756302): {2, False, 'full'},\n",
       " (99.93263898075874, 99.92987012490183): {2, False, 'tied'},\n",
       " (99.95150540934567, 99.95150851608659): {2, False, 'diag'},\n",
       " (99.86495075170222, 99.86282165749554): {2, False, 'spherical'},\n",
       " (99.74949729922261, 99.75188220846508): {3, False, 'full'},\n",
       " (99.81223158195307, 99.80756789724285): {3, False, 'tied'},\n",
       " (99.64331331754973, 99.64693625918024): {3, False, 'diag'},\n",
       " (99.71357609508574, 99.71552151080657): {3, False, 'spherical'}}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Rscore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Artificial Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,Y_train,Y_test,scaler = import_dataset(scaleMethod='MinMax')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_28 (Dense)             (None, 100)               800       \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 100)               400       \n",
      "_________________________________________________________________\n",
      "dense_29 (Dense)             (None, 100)               10100     \n",
      "_________________________________________________________________\n",
      "dense_30 (Dense)             (None, 100)               10100     \n",
      "_________________________________________________________________\n",
      "dense_31 (Dense)             (None, 100)               10100     \n",
      "_________________________________________________________________\n",
      "dense_32 (Dense)             (None, 100)               10100     \n",
      "_________________________________________________________________\n",
      "dense_33 (Dense)             (None, 100)               10100     \n",
      "_________________________________________________________________\n",
      "dense_34 (Dense)             (None, 4)                 404       \n",
      "=================================================================\n",
      "Total params: 52,104\n",
      "Trainable params: 51,904\n",
      "Non-trainable params: 200\n",
      "_________________________________________________________________\n",
      "Epoch 1/100\n",
      "200/200 [==============================] - 31s 156ms/step - damping_factor: 10000000000.0000 - attempts: 1.0000 - loss: 0.0881 - accuracy: 0.2394\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x19d04fbd250>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(100, input_dim=X_train.shape[1],\n",
    "                kernel_initializer=initializers.RandomUniform(minval=0, maxval=1, seed=None), \n",
    "                activation='relu',\n",
    "                kernel_regularizer=regularizers.l2(1e-4)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(100, activation='relu'))\n",
    "model.add(Dense(100, activation='relu'))\n",
    "model.add(Dense(100, activation='relu'))\n",
    "model.add(Dense(100, activation='relu'))\n",
    "model.add(Dense(100, activation='relu'))\n",
    "model.add(Dense(Y_train.shape[1], activation='relu'))\n",
    "opt = optimizers.Adam(learning_rate=0.001)\n",
    "model.compile(loss='mean_squared_error', optimizer=opt, metrics=['accuracy'])\n",
    "model.summary()\n",
    "\n",
    "model_wrapper = lm.ModelWrapper(\n",
    "    tf.keras.models.clone_model(model))\n",
    "\n",
    "model_wrapper.compile(\n",
    "    optimizer=SGD(learning_rate=1.0),\n",
    "    loss=lm.MeanSquaredError(),metrics=['accuracy'])\n",
    "\n",
    "model_wrapper.fit(X_train, Y_train, epochs=100, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train.reshape(X_train.shape[0],X_train.shape[1],1)\n",
    "Y_train = Y_train.reshape(Y_train.shape[0],Y_train.shape[1],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12750, 1, 1)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_train[:,[1],:].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "128/128 [==============================] - 0s 2ms/step - loss: 0.0583 - accuracy: 0.6340\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x21a40cc37c0>"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linear_model = LinearModel()\n",
    "dnn_model = Sequential([Dense(units=100),\n",
    "                        Dense(units=150),\n",
    "                        Dense(units=150),\n",
    "                        Dense(units=150),\n",
    "                        Dense(units=150),\n",
    "                        Dense(units=150),\n",
    "                             Dense(units=Y_train.shape[1])])\n",
    "combined_model = WideDeepModel(linear_model, dnn_model)\n",
    "combined_model.compile(['sgd', 'adam'], 'mse', metrics=['accuracy'])\n",
    "# define dnn_inputs and linear_inputs as separate numpy arrays or\n",
    "# a single numpy array if dnn_inputs is same as linear_inputs.\n",
    "combined_model.fit([X_train, X_train], Y_train, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "Ytr_pred = combined_model.predict([X_train,X_train])\n",
    "Yts_pred = combined_model.predict([X_test,X_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "true = inverseTransform(scaler,X_train,Y_train)\n",
    "pred = inverseTransform(scaler,X_train,Ytr_pred)\n",
    "\n",
    "error_tr = rmse(true[:,7:],pred[:,7:])\n",
    "\n",
    "true = inverseTransform(scaler,X_test,Y_test)\n",
    "pred = inverseTransform(scaler,X_test,Yts_pred)\n",
    "\n",
    "error = rmse(true[:,7:],pred[:,7:])\n",
    "magnitude = errorMagnitude(true[:,7:],pred[:,7:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.6985740824588005"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "error_tr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.0009603500366210938, 6.815523147583006)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "magnitude"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
